# -*- coding: utf-8 -*-
import scrapy
from scrapy.http import Request,FormRequest
import urllib.request
from csdn_edu.items import CsdnEduItem

class CsdnSpider(scrapy.Spider):
    name = 'csdn'
    allowed_domains = ['csdn.net']
    #start_urls = ['http://csdn.net/']
    header = {"User-Agent":"Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/62.0.3202.62 Safari/537.36"}

    def start_requests(self):
        return [Request("https://passport.csdn.net/account/login",meta={"cookiejar":1},callback=self.login)]

    def login(self, response):
        captcha = response.xpath("//div[@id='yanzheng']").extract()
        if len(captcha) > 0:
            print("有验证码")
            localpath = "D:/python/csdn/yanzhengma/captcha.png"
            urllib.request.urlretrieve(captcha[0],filename = localpath)
            captcha_value = input("请到"+localpath+"查看验证码")
            data = {
                "captcha-solution":captcha_value,
                "username":'baoyaoben@21cn.com',
                "password":'baoyaoben@21cn.com',
            }
            print("登录中...")
            return [FormRequest.from_response(response,
                                              meta = {"cookiejar": response.meta["cookiejar"]},
                                              headers = self.header,
                                              formdata = data,
                                              callback = self.next,
                                              )]
        else:
            data = {
                "username": 'baoyaoben@21cn.com',
                "password": 'baoyaoben@21cn.com',
            }
            print("登录中...")
            return [FormRequest.from_response(response,
                                              meta = {"cookiejar": response.meta["cookiejar"]},
                                              headers = self.header,
                                              formdata = data,
                                              callback = self.next,
                                              )]
        pass

    def next(self, response):
        print("登录成功")
        with open('D:\\python\pythonQuanZhanXueXi\scrapy\loginOK.html','wb') as file:
            file.write(response.body)
        for i in range(1,244):
            yield Request('https://edu.csdn.net/courses/p'+str(i),
                          meta = {"cookiejar": response.meta["cookiejar"]},
                          headers = self.header,
                          callback = self.parsecourses,
                          dont_filter = True
                          )

    def parsecourses(self, response):
        # print('获取所有录播课程链接')
        # with open('logintest.html','wb') as file:
        #     file.write(response.body)
        urllist = response.xpath("//div[@class='course_dl_list']/a/@href").extract()
        # print(urllist)
        for url in urllist:
            print(url)
            yield Request(url,
                    meta = {"cookiejar": response.meta["cookiejar"]},
                    headers = self.header,
                    callback = self.parse,
                    dont_filter=True,
                    )

    def parse(self,response):
        # print("获取课程信息")
        item = CsdnEduItem()
        item["link"] = response.url
        item["title"] = response.xpath("//div[starts-with(@class,'info_right')]/h1/a/text()").extract()
        item["number"] = response.xpath("//span[@class='num']/text()").extract()
        item["price"] = response.xpath("normalize-space(//span[@class='money']/text())").extract()
        item["time"] = response.xpath("//span[@class='pinfo']/text()").extract()
        yield item


        
